package advertising.etl

import advertising.entity.RawTabData
import advertising.handler.TypeCastHandler
import advertising.sql.EtlSQL
import org.apache.spark.SparkConf
import org.apache.spark.sql.{Dataset, SparkSession}

//  .--,       .--,
// ( (  \.---./  ) )
//  '.__/o   o\__.'
//     {=  ^  =}
//      >  -  <
//     /       \
//    //       \\
//   //|   .   |\\
//   "'\       /'"_.-~^`'-.
//      \  _  /--'         `
//    ___)( )(___
//   (((__) (__)))    高山仰止, 景行行止; 虽不能至, 心向往之.
//                    High mountains, scenery. Although not to, the heart yearns for it.

/**
  * 处理原始日志
  */
object SimpleCleanData2Hive {

  def main(args: Array[String]): Unit = {

    if(args.length < 1){
      System.err.println("Invalid parameter input~")
      System.exit(1)
    }

    val input = args(0)

    val conf = new SparkConf()
      .setAppName("SimpleCleanDataToHive")

    val spark = SparkSession.builder.config(conf).enableHiveSupport().getOrCreate()
    spark.sparkContext.setLogLevel("WARN")
    val linesDS = spark.read.textFile(input)
    import spark.implicits._
    linesDS.map(_.split(","))
      .filter(_.length >= 85)
      .map(TypeCastHandler.arr2AdRawInfo(_))
      .createOrReplaceTempView("cleanedAdRawData")


    spark.sql(EtlSQL.cleanedAdRawInfo2Hive)
//    cleanedAdRawInfo.write.parquet(output)
    spark.stop()
  }

}
